package com.gxljc.bear.crawler.itaogao.ccstock;

import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;

import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONObject;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;
import com.gxljc.bear.crawler.itaogao.NewsBaseCrawler;
import com.gxljc.bear.crawler.itaogao.NewsBaseTable;
import com.gxljc.bear.crawler.util.DateUtil;
import com.gxljc.bear.crawler.util.HtmlUtil;
import com.gxljc.bear.crawler.util.JsoupUtil;

/**
 * 证券日报
 * 
 * @author Administrator->haitao
 * @since 2021年9月14日 下午5:51:54
 */
public class CcstockDetailCrawler extends NewsBaseCrawler {

	private static final long serialVersionUID = -1084336561181882139L;
	final static Logger LOG = Logger.getLogger(CcstockDetailCrawler.class);
	final static String LIST_FORMAT = "http://zqrb.ccstock.cn/html/%s/node_2.htm";
	final static String URL_FORMAT = "http://zqrb.ccstock.cn/html/%s/%s";

	public CcstockDetailCrawler() {
		super("t_newspaper_ccstock", DolphinCrawlerConsts.MongoDBName.ITAOGAO.getName());
	}

	@Override
	protected List<String> genSeeds() {
		LOG.info("--- genSeed of ccstock ----");
		List<String> seeds = new ArrayList<>();
		try {
			for (String date : genDatesTillNow(DEFAULT_DATE_FORMAT)) {
				// fetch url list by parser
				String url = String.format(LIST_FORMAT, date);
				byte[] bytes = fetch(url, proxy);
				if (bytes == null) {
					continue;
				}
				String html = new String(bytes);
				List<String> hrefList = JsoupUtil.extractAttrList(html, ".vote_content12px", "href");
				List<String> urls = new ArrayList<>();
				for (String s : hrefList) {
					urls.add(String.format(URL_FORMAT, date, s));
				}
				seeds.addAll(urls);
			}

		} catch (Exception e) {
			e.printStackTrace();
		}
		LOG.info("total size=" + seeds.size());
		return seeds;
	}

	@Override
	protected List<String> genNewSeeds() {
		LOG.info("--- genNewSeed of ccstock ----");
		List<String> seeds = new ArrayList<>();
		String date = DateUtil.dateFormat(new Date(), "yyyy-MM/dd");
		// fetch url list by parser
		String url = String.format(LIST_FORMAT, date);
		String html = "";
		try {
			byte[] bytes = fetch(url, proxy);
			if (bytes == null) {
				LOG.error(String.format("fetch empty for url=", url));
				return seeds;
			}
			html = new String(bytes);
			List<String> hrefList = JsoupUtil.extractAttrList(html, ".vote_content12px", "href");

			List<String> urls = new ArrayList<>();
			for (String s : hrefList) {
				urls.add(String.format(URL_FORMAT, date, s));
			}
			seeds.addAll(urls);
		} catch (Exception e) {
			e.printStackTrace();
		}

		LOG.info("total size=" + seeds.size());
		return seeds;
	}

	@Override
	protected Pair<NewsBaseTable, byte[]> genTable(String url) {

		NewsBaseTable table = new NewsBaseTable();

		// fetch from template
		Pair<String, byte[]> pair = fetResult(url, proxy);
		if (pair == null) {
			return new Pair<NewsBaseTable, byte[]>(table, "".getBytes());
		}
		String result = pair.first;
		byte[] html = pair.second;
		if (result != null) {
			JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
			if (jsonResult != null) {

				table.setPublishDate(extractDate(url));
				String title = (String) jsonResult.get("title");
				if (!StringUtil.isNullOrEmpty(title)) {
					table.setTitle(title);
				}
			}
		}
		// parse page by Jsoup
		if (html != null) {
			String htmlPage = new String(html);
			table = extractTable(url, htmlPage);
		}
		table.setPublishDate(extractDate(url));
		// set default value
		table.setId(StringUtil.genMD5Val(url));
		table.setCrawlUrl(url);
		table.setCrawlDate(new Date());
		table.setCrawlFlag(1);
		return new Pair<>(table, html);

	}

	private NewsBaseTable extractTable(String url, String html) {
		NewsBaseTable table = new NewsBaseTable();

		// fetch images
		String css3 = ".content table tbody tr td div table tbody tr td table tbody tr td img";
		List<String> images = JsoupUtil.extractAttrList(html, css3, "src");
		table = processImages(url, table, images);

		// fetch content
		String css4 = "div founder-content p";
		List<String> contents = JsoupUtil.extractList(html, css4);
		List<String> nContents = new LinkedList<>();
		for (String s : contents) {
			if ("&nbsp;".equals(s) || s.isEmpty()) {
				continue;
			}
			nContents.add(s.replaceAll("&nbsp;", ""));
		}
		if (validateObjects(nContents)) {
			table.setContent(genContent(nContents, table.getImages()));
		}
		
		List<String> titles = JsoupUtil.extractList(html, "tr td .h1");
		if(validateObject(titles)){
			table.setTitle(titles.get(0));
		}
		return table;
	}

	public static void main(String[] args) throws Exception {
		String request = "http://zqrb.ccstock.cn/html/2016-09/26/node_2.htm";
		request = "http://zqrb.ccstock.cn/html/2016-08/23/node_2.htm";
		String html = new String(HtmlUtil.fetch(request));
		String css = ".vote_content12px";
		List<String> lists = JsoupUtil.extractAttrList(html, css, "href");
		System.out.println(lists);
		System.out.println("size=" + lists.size());

		css = ".content table tbody tr td div table tbody tr td table tbody tr td img";
		request = "http://zqrb.ccstock.cn/html/2016-09/26/content_213773.htm";
		html = new String(HtmlUtil.fetch(request));
		System.out.println(JsoupUtil.extractAttrList(html, css, "src"));
		
		request = "http://zqrb.ccstock.cn/html/2016-11/15/content_227926.htm";
		html = new String(HtmlUtil.fetch(request));
		List<String> titles = JsoupUtil.extractList(html, "tr td .h1");
		System.out.println("titles="+titles);
	}

}
